import com.listener.pojo.Ability;
import com.listener.pojo.Move;
import com.listener.pojo.Pokemon;
import org.apache.ibatis.io.Resources;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.session.SqlSessionFactoryBuilder;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class CrawlingData {
    //wiki的根路径
    private SqlSession sqlSession;
    private Reader reader;

    @Before
    public void init(){
        String resource="mybatis-config.xml";
        Reader reader= null;
        try{
            reader= Resources.getResourceAsReader(resource);
            SqlSessionFactory sqlMapper=new SqlSessionFactoryBuilder().build(reader);
            sqlSession=sqlMapper.openSession();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    String rootUrl="https://wiki.52poke.com";
    //根据url获取网页的json
    private String getJsonByUrl(String url) {
        try {
            url = url;
            Connection conn = Jsoup.connect(url)
                    .method(Connection.Method.GET)
                    .ignoreContentType(true);
            //            conn.data("param", url);如果需要添加特定参数 此时不需要
            Connection.Response response = conn.execute();
            if (response.statusCode() == 200) {
                String json = new String(response.bodyAsBytes(), "UTF-8");
                return json;
            }else{
                return "请求异常";
            }
        }catch (Exception e){
            e.printStackTrace();
            return "编码异常";
        }
    }
    //根据正则表达式返回Matcher
    public Matcher getMatcherByRegex(String parent, String regex){
        Pattern pattern = Pattern.compile(regex);
        return pattern.matcher(parent);
    }
    //根据正则表达式查找第一个符合条件的字符串
    public String getStringByRegex(String parent,String regex)
    {
        Pattern pattern = Pattern.compile(regex);
        Matcher matcher = pattern.matcher(parent);
        if(matcher.find()){
            return matcher.group();
        }else{
            return null;
        }
    }
    //爬取宝可梦全国图鉴一级数据
    @Test
    public void crawlingPokemon(){
        String pokemonsParentJson=getJsonByUrl(rootUrl+ "/wiki/宝可梦列表（按全国图鉴编号）");
        Matcher pokemonMatcher =getMatcherByRegex(pokemonsParentJson,
                "<tr data-type=\".{0,6}\">[\\s\\S\\n]{0,2000}?</tr>");
        String nowId,lowId=null;
        while(pokemonMatcher.find()){
            String pokemonJson = pokemonMatcher.group();
            String id=getStringByRegex(pokemonJson,"(?<=<td>).{0,10}(?=\\n)");
            nowId=id;
            if(nowId.equals(lowId)){
                continue;
            }
            String name=getStringByRegex(pokemonJson,"(?<=title=\").{0,20}?(?=\")");
            String childUrl=rootUrl+getStringByRegex(pokemonJson,"(?<=href=\").{0,200}?(?=\")");
            String typeString=getStringByRegex(pokemonJson,"(?<=<tr data-type=\").{0,20}(?=\">)");
            String[] types = typeString.split(":");
            String type = types[0];//第一属性
            String type1=types.length>=2?types[1]:null;
            Pokemon pokemon=new Pokemon();
            System.out.print("编号:【"+id+"】\t");
            pokemon.setPokemonId(id);
            System.out.print("名称:【"+name+"】\t");
            pokemon.setPokemon(name);
            if(type1==null){
                System.out.print("属性:【"+type+"】\t");
                pokemon.setType(type);
            }else{
                System.out.print("第一属性:【"+type+"】\t");
                pokemon.setType(type);
                System.out.print("第二属性:【"+type1+"】\t");
                pokemon.setType1(type1);
            }
            System.out.println("子链接为:【"+childUrl+"】\t");
            pokemon.setWikiUrl(childUrl);
            Integer count=sqlSession.insert("insertPokemon",pokemon);
            if(count!=1){
                System.out.println("异常");
                return;
            }
            lowId=nowId;
        }
    }
    @Test
    public void crawlingPokemonWiki() {//爬取宝可梦的详情页信息
        List<Pokemon> pokemons = sqlSession.selectList("selectPokemons");
        Integer index=0;
        for (Pokemon pokemon : pokemons) {
            if(pokemon.getId()<=6127)continue;
            if(index++==100)break;
            String pokemonWikiUrl=pokemon.getWikiUrl();
            //宝可梦详情页的json
            String pokemonWikiJson=getJsonByUrl(pokemonWikiUrl);
            String pictureUrl="https:"+ getStringByRegex(pokemonWikiJson,
                    "(?<=data-url=\").{0,200}?(?=\"[\\s\\S\\n]{0,500}?<div>官方绘图</div>)");
            //宝可梦图片网址
            pokemon.setPicture(pictureUrl);
            String hidden_ability=getStringByRegex(pokemonWikiJson,"(?<=title=\".{0,10}?（特性）\">).*(?=</a><br /><small>隱藏特性</small>)");
            if(hidden_ability!=null){
                //隐藏特性
                pokemon.setHiddenAbility(hidden_ability);
            }
            String abilityJson=getStringByRegex(pokemonWikiJson,"(?<=title=\"特性\")[\\s\\S\\n]{0,1000}(?=</tbody>)");
            Matcher abilityMatcher=getMatcherByRegex(abilityJson,"(?<=title=\".{0,10}（特性）\">).*?(?=</a>)");
            int abilityIndex=1;
            while(abilityMatcher.find()) {
                String type=abilityMatcher.group();
                if(type.equals(hidden_ability))
                    break;
                if(abilityIndex==1){
                    //第一特性
                    pokemon.setAbility(type);
                }else{
                    //第二特性
                    pokemon.setAbility1(type);
                }
                abilityIndex++;
            }
            //六维
            pokemon.setHp(Integer.parseInt(getStringByRegex(
                    pokemonWikiJson,"(?<=ＨＰ.{0,50}\"float:right\">).{0,10}(?=<)")));
            pokemon.setAttack(Integer.parseInt(getStringByRegex(
                    pokemonWikiJson,"(?<=攻击.{0,50}\"float:right\">).{0,10}(?=<)")));
            pokemon.setDefense(Integer.parseInt(getStringByRegex(
                    pokemonWikiJson,"(?<=防御.{0,50}\"float:right\">).{0,10}(?=<)")));
            pokemon.setSpecialAttack(Integer.parseInt(getStringByRegex(
                    pokemonWikiJson,"(?<=特攻.{0,50}\"float:right\">).{0,10}(?=<)")));
            pokemon.setSpecialDefense(Integer.parseInt(getStringByRegex(
                    pokemonWikiJson,"(?<=特防.{0,50}\"float:right\">).{0,10}(?=<)")));
            pokemon.setSpeed(Integer.parseInt(getStringByRegex(
                    pokemonWikiJson,"(?<=速度.{0,50}\"float:right\">).{0,10}(?=<)")));
            int count=sqlSession.update("supplePokemon",pokemon);
            if(count!=1){
                System.out.println("异常");
                return;
            }
            //可学会的招式
            String pokemonMoveJson=getStringByRegex(pokemonWikiJson,"(?<=可学会的招式[\\s\\S\\n]{0,5000}?等级)[\\s\\S\\n]*?(?=</tbody>)");
            Matcher pokemonMoveMatcher=getMatcherByRegex(pokemonMoveJson,"(?<=<tr class=\"at-c bgwhite\">)[\\s\\S\\n]*?(?=</tr>)");
            while(pokemonMoveMatcher.find()){
                String moveJson= pokemonMoveMatcher.group();
                Map<String,String> pokemonMoveEarn=new HashMap<String,String>();
                pokemonMoveEarn.put("pokemon",pokemon.getPokemon());
                pokemonMoveEarn.put("move",getStringByRegex(moveJson,"(?<=title=\").{0,100}(?=（招式）)"));
                pokemonMoveEarn.put("level",getStringByRegex(moveJson,"(?<=<td>).{0,10}(?=[\\s\\S\\n]{0,10}</td>)"));
                if(pokemonMoveEarn.get("level")==null){
                    pokemonMoveEarn.put("level",getStringByRegex(moveJson,"(?<=>).{0,10}(?=[\\s\\S\\n]{0,10}</td>)"));
                }
                count=sqlSession.insert("insertPokemonMoveEarn",pokemonMoveEarn);
                if(count!=1){
                    System.out.println("异常");
                    return;
                }
            }
            System.out.println(pokemon);
        }
    }
    //爬取招式列表
    @Test
    public void crawlingMove(){
        String MovesParentJson=getJsonByUrl(rootUrl+ "/wiki/招式列表");
        Matcher MoveMatcher =getMatcherByRegex(MovesParentJson,
                "<tr data-type=[\\s\\S\\n]{0,2000}?</tr>");
        while(MoveMatcher.find()){
            Move move=new Move();
            String MoveChildJson=MoveMatcher.group();
            move.setMove(getStringByRegex(MoveChildJson,"(?<=title=\").{0,40}?(?=（招式）)"));
            Matcher MoveChildMatcher=getMatcherByRegex(MoveChildJson,"(?<=<td>).*?(?=\\n)");
            int MoveChildIndex=0;
            if(move.getMove()!=null){//说明不是Z招式
                while(MoveChildMatcher.find()){
                    MoveChildIndex++;
                    String text=MoveChildMatcher.group();
                    if(MoveChildIndex==1){
                        move.setMoveId(text);
                    }else if(MoveChildIndex==4){
                        move.setPower(text);
                    }else if(MoveChildIndex==5){
                        move.setAccuracy(text);
                    }else if(MoveChildIndex==6){
                        move.setPP(text);
                    }else if(MoveChildIndex==7){
                        move.setDescription(text);
                    }
                }
            }else {
                move.setMove(getStringByRegex(MoveChildJson,"(?<=title=\".{3,10}\">).{0,20}(?=</a>)"));
                while(MoveChildMatcher.find()){
                    MoveChildIndex++;
                    String text=MoveChildMatcher.group();
                    if(MoveChildIndex==1){
                        move.setMoveId(text);
                    }else if(MoveChildIndex==3){
                        move.setPower(text);
                    }else if(MoveChildIndex==4){
                        move.setAccuracy(text);
                    }else if(MoveChildIndex==5){
                        move.setPP(text);
                    }else if(MoveChildIndex==6){
                        move.setDescription(text);
                    }
                }
            }
            move.setWikiUrl(rootUrl+getStringByRegex(MoveChildJson,"(?<=href=\").{0,200}?(?=\")"));
            move.setType(getStringByRegex(MoveChildJson,"(?<=data-type=\").{0,40}?(?=\")"));
            move.setCategory(getStringByRegex(MoveChildJson,"(?<=data-category=\").{0,40}?(?=\")"));
            int count=sqlSession.insert("insertMove",move);
            if(count!=1){
                System.out.println("异常");
                return;
            }
        }
    }

    //爬取特性列表
    @Test
    public void crawlingAbility() {
        String AbilityParentJson=getJsonByUrl(rootUrl+ "/wiki/特性列表");
        Matcher AbilityMatcher =getMatcherByRegex(AbilityParentJson,
                "<tr>[\\s\\S\\n]{0,30}<td width=\"45\">[\\s\\S]*?</tr>");
        while(AbilityMatcher.find()){
            String AbilityChildJson =AbilityMatcher.group();
            Ability ability =new Ability();
            ability.setAbilityId(getStringByRegex(AbilityChildJson,"(?<=<td width=\"45\">)\\d+"));
            ability.setWikiUrl(rootUrl+getStringByRegex(AbilityChildJson,"(?<=href=\").*?(?=\")"));
            ability.setAbility(getStringByRegex(AbilityChildJson,"(?<=title=\").*?(?=（特性）)"));
            ability.setDescription(getStringByRegex(AbilityChildJson,"(?<=<td class=\"at-l\">).*?(?=\n)"));
            int count=sqlSession.insert("insertAbility",ability);
            if(count!=1){
                System.out.println("异常");
                return;
            }
        }
    }

    //读取流
    public static byte[] readInputStream(InputStream inStream) throws Exception {
        ByteArrayOutputStream outStream = new ByteArrayOutputStream();
        //创建一个Buffer字符串
        byte[] buffer = new byte[6024];
        //每次读取的字符串长度
        int len;
        //使用一个输入流从buffer里把数据读取出来
        while ((len = inStream.read(buffer)) != -1) {
            //用输出流往buffer里写入数据，中间参数代表从哪个位置开始读，len代表读取的长度
            outStream.write(buffer, 0, len);
        }
        //关闭输入流
        inStream.close();
        //把outStream里的数据写入内存
        return outStream.toByteArray();
    }
    //下载文件的工具函数
    public void downloadFile(String fileUrl,String downloadPath) throws Exception {
        URL url = new URL(fileUrl);
        //打开连接
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        //设置请求方式为"GET"
        conn.setRequestMethod("GET");
        conn.setRequestProperty("Content-Type", "application/json");
//            conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
        conn.setRequestProperty("connection", "keep-alive");
        conn.setRequestProperty("User-Agent", "Mozilla/4.76");
        conn.setUseCaches(false);//设置不要缓存
        conn.setInstanceFollowRedirects(true);
        conn.setDoOutput(true);
        conn.setDoInput(true);
        //超时响应时间为10秒
        conn.setConnectTimeout(10 * 1000);
        //通过输入流获取图片数据
        InputStream is = conn.getInputStream();
        //得到图片的二进制数据，以二进制封装得到数据，具有通用性
        byte[] data = readInputStream(is);
        //创建一个文件对象用来保存图片
        String folderPath=downloadPath.substring(0,downloadPath.lastIndexOf("\\"));
        File folder=new File(folderPath);
        if(!folder.exists()){
            folder.mkdirs();
        }
        File imageFile = new File(downloadPath);
        //创建输出流
        FileOutputStream outStream = new FileOutputStream(imageFile);
        //写入数据
        outStream.write(data);
        //关闭输出流，释放资源
        outStream.close();
    }
    //根据网址和id下载图片
    public String downloadByUrlAndPokemonId(String url,String pokemon_id){
        String projectDirectory = System.getProperty("user.dir");
        String downloadPath=projectDirectory+"\\src\\main\\webapp\\img\\pokemonPicture\\pokemon"+pokemon_id+".png";
        try {
            downloadFile(url,downloadPath);
            return "img/pokemonPicture/pokemon"+pokemon_id+".png";
        }catch (java.net.SocketTimeoutException e){
            System.out.println(downloadPath+"路径"+url+"网址超时");
            return null;
        } catch (Exception e) {
            e.printStackTrace();
            return null;
        }
    }
    //根据图片链接下载图片到本地并更换为本地路径
    @Test
    public void downloadPicture(){
        List<Pokemon> pokemons = sqlSession.selectList("selectPokemons");
        for (Pokemon pokemon : pokemons) {
//            if(pokemon.getId()<=5776){continue;}
            String picturePath= downloadByUrlAndPokemonId(
                    pokemon.getPicture(),pokemon.getPokemonId().substring(1));//去掉了#
            if(picturePath!=null){
                pokemon.setPicture(picturePath);
                int count=sqlSession.insert("updatePokemonPicture",pokemon);
                if(count!=1){
                    System.out.println("mysql更新图片路径异常");
                    return;
                }else {
                    System.out.println(pokemon.getPokemonId()+pokemon.getPokemon()+"图片下载成功");
                }
            }else {
                System.out.println("下载图片异常");
                return;
            }
        }
    }
    @After
    public void destroy(){
        sqlSession.commit();
        sqlSession.close();
    }
}
